import os, tarfile, sys
import os.path as osp
import shutil
import time
from collections import defaultdict
import numpy as np
import cv2
import json


# from tool import filesystem 
# # export PYTHONPATH=$PYTHONPATH:`pwd`
# set PYTHONPATH=%PYTHONPATH%;./

# 递归获取文件夹下所有符合条件的文件路径.
def get_all_filepath(root_dir_path, extension_tag=[], recursive = True):
    """
    Args:
        root_dir_path: 文件根目录.
        extension_tag: 文件后缀名组成的列表, Example: ["jpg", "png", "bmp"]
        如果为空 则获取所有
    Returns:
        特定文件夹下所有符合条件的文件路径.
    """
    def check_endswith(new_path, extension_tag):
        if len(extension_tag) == 0:
            return True
        for s in extension_tag:
            if new_path.lower().endswith(s):
                return True
        return False

    paths = []
    for f in os.listdir(root_dir_path):
        new_path = root_dir_path + os.sep + f
        if os.path.isfile(new_path) and check_endswith(new_path,  extension_tag):
            paths.append(new_path)
        elif os.path.isdir(new_path) and recursive:
            temp_list = get_all_filepath(new_path, extension_tag)
            paths.extend(temp_list)
    return paths

# 递归获取文件夹下所有最后一层子文件夹.
def get_last_dir(root_dir_path, recursive_level = -1):
    """
    Args:
        root_dir_path: 文件夹根目录.
    Returns:
        特定文件夹下所有的文件夹路径.
    """
    paths = []
    for f in os.listdir(root_dir_path):
        new_path = root_dir_path + os.sep + f
        if osp.isdir(new_path):
            if recursive_level > 1 or recursive_level < 0:
                ps = get_last_dir(new_path, recursive_level-1)
                paths.extend(ps)
            else:
                paths.append(new_path)
    if len(paths) == 0:
        paths.append(root_dir_path)
    return paths
# 递归获取文件夹下所有via.json信息.
def get_all_via_data_in_path_dir(root_dir_path, extension_tag=[]):
    """
    Args:
        root_dir_path: 文件根目录.
        extension_tag: 文件后缀名组成的列表, Example: ["jpg", "png", "bmp"]
    Returns:
        特定文件夹下所有符合条件的文件路径.
    """
    def check_endswith(new_path, extension_tag):
        if len(extension_tag) == 0:
            return True
        for s in extension_tag:
            if new_path.endswith(s):
                return True
        return False

    datadict = dict()
    for f in os.listdir(root_dir_path):
        new_path = root_dir_path + os.sep + f
        if os.path.isfile(new_path) and check_endswith(new_path,  extension_tag):
            with open(new_path, "r") as rf:
                data = json.loads(rf.read())
                for key in data.keys():
                    datadict[root_dir_path + os.sep +data[key]["filename"]] = data[key]
        elif os.path.isdir(new_path):
            temp_list = get_all_via_data_in_path_dir(new_path, extension_tag)
            datadict.update(temp_list)
    return datadict


def remove_same_files(std_dir_path, remove_dir):
    """
    删除系统中相同路径下的文件
    """
    std_files = get_all_filepath(std_dir_path)

    for f in std_files:
        new_path = f.replace(std_dir_path, remove_dir)
        os.remove(new_path)
    
def clear_dir_by_no_via_json(data_root, via_name=""):
    for name in os.listdir(data_root):
        via_dir = data_root + os.sep + name
        if os.path.exists(via_dir + os.sep + via_name):
            continue
        shutil.rmtree(via_dir)

def read_via_file(via_path, allow_empty=False):
    """
    读取 via 文件，返回图片绝对路径
    @allow_empty: 允许保留无标签图片
    """
    root_dir_path = os.path.dirname(via_path)
    datadict = dict()
    with open(via_path, "r") as rf:
        data = json.loads(rf.read())
        for key in data.keys():
            if not allow_empty and len(data[key]["regions"]) == 0:
                print(data[key]["filename"], " regions is None")
                continue
            datadict[root_dir_path + os.sep +data[key]["filename"]] = data[key]
    return datadict

# 递归取一个文件夹下所有文件到目标文件夹
count = 0
def move_all_to_one(all_dir_path, dest_dir):
    global count
    for f in os.listdir(all_dir_path):
        new_path = all_dir_path + os.sep + f
        if os.path.isfile(new_path):
            dest_path = dest_dir + os.sep + str(count) + '_' + str(count) + '_' + os.path.basename(new_path)
            print(dest_path)
            count += 1
            shutil.move(new_path, dest_path)
        elif os.path.isdir(new_path):
            move_all_to_one(new_path, dest_dir)
# all_dir_path = r'C:\Users\vanlance\Desktop\2\2016\new'
# dest_dir = r'C:\Users\vanlance\Desktop\2\2016\out'
# move_all_to_one(all_dir_path, dest_dir)

# 递归取一个文件夹下所有文件到目标文件夹
def move_all_to_one2(dir_path, dest_dir, per_dir=4):
    count = 0
    for data_dir in get_last_dir(dir_path):
        files = [osp.join(data_dir, f) for f in os.listdir(data_dir)]
        step = len(files) // per_dir
        for i in range(0, len(files), step):
            shutil.copy(files[i], osp.join(dest_dir, osp.basename(files[i])))
            # os.remove(files[i])

# 将 space 替换成 _
def replace_char(dir_path):
    for f in get_all_filepath(dir_path, [".jpg", '.png']):
        name = osp.basename(f)
        print(name)
        if " " in name:
            name = name.replace(" ", "_")
            shutil.move(f, osp.join(osp.dirname(f), name))

# 递归取一个文件夹下所有文件到目标文件夹，并将文件目录信息保留至文件名

def move_all_to_one_keep_name(all_dir_path, dest_dir, name_prefix=''):
    for f in os.listdir(all_dir_path):
        new_name_prefix = name_prefix + '_' + f
        new_path = all_dir_path + os.sep + f
        if os.path.isfile(new_path) and f.endswith("pdf"):
            dest_path = dest_dir + os.sep + new_name_prefix
            shutil.copyfile(new_path, dest_path)
        elif os.path.isdir(new_path):
            move_all_to_one_keep_name(new_path, dest_dir, new_name_prefix)

#重命名
def rename_files_from_dir(dir_path):
    start_count = 500
    for f in os.listdir(dir_path):
        file_ = dir_path + os.sep + f
        os.rename(file_, dir_path + os.sep + "{}_.".format(start_count)  + f.split(".")[-1]  )   # 
        start_count += 1

#重命名
def rename_files_from_dir_save_to_map(dir_path):
    map_txt = open(osp.join(osp.dirname(dir_path), "map.txt"), "w")
    start_count = 600
    for f in os.listdir(dir_path):
        file_ = dir_path + os.sep + f
        os.rename(file_, dir_path + os.sep + "{}.".format(start_count)  + f.split(".")[-1]  )   # 
        map_txt.write("{} {}\n".format(start_count, f))
        start_count += 1


#剔除一部分
count_2 = 0
def move_some_img_condion(dir_path):
    global count_2
    for f in os.listdir(dir_path):
        if len(f.split('out')) == 1:
            new_f = dir_path + os.sep + '_out.'.join(f.split('.'))
            if not os.path.exists(new_f):
                to_path = dir_path + os.sep + f
                shutil.move(to_path, to_path.replace('out', 'pos'))
# dir_path = r'C:\Users\vanlance\Desktop\2\2016\out'
# move_some_img_condion(dir_path)

# 重命名---成对
def rename_by_pair(dest_dir):
    name_list = []
    xml_list = []
    for name in os.listdir(dest_dir):
        if name[-3:] == 'jpg':
            name_list.append(name.split('_') + [name])
        else:
            xml_list.append(name.split('_') + [name])
    name_list = sorted(name_list)
    xml_list = sorted(xml_list)
    
    # length = len(name_list)
    for idx, name in enumerate(name_list):
        value = int(name[0]) +1 
        for xml in xml_list:
            value1 = int(xml[0])
            if value == value1:
                os.rename(os.path.join(dest_dir, name[-1]), os.path.join(dest_dir, str(idx) + '.jpg'))
                os.rename(os.path.join(dest_dir, xml[-1]), os.path.join(dest_dir, str(idx) + '.xml'))
                print(idx)
                continue

# make train test by pair jpg and xml
def make_train_test_by_pair(train_dir, test_dir):
    files = [os.path.join(train_dir, p)  for p in os.listdir(train_dir) if p[-3:] == 'jpg']
    
    np.random.shuffle(files)
    np.random.shuffle(files)

    length = len(files)

    for idx, img in enumerate(files):
        if idx < length * 0.17:
            shutil.move(img, os.path.join(test_dir, os.path.basename(img)))
            shutil.move(img.replace('jpg', 'xml'), os.path.join(test_dir, os.path.basename(img).replace('jpg', 'xml')))

#删除符合条件的文件

def remove_files_from_dir(dir_path):
    for f in os.listdir(dir_path):
        file_ = dir_path + os.sep + f
        if file_.endswith('png'):
            os.remove(file_)
# dir_path = r'C:\Users\vanlance\Desktop\2\2016\out'
# rename_files_from_dir(dir_path)

# 将一个文件夹下的所有文件随机均分配至多个文件夹
def equal_distribution(src_dir, per_dir_num=500, extension_tag=[".jpg"]):
    all_files = get_all_filepath(src_dir, extension_tag)

    np.random.shuffle(all_files)
    np.random.shuffle(all_files)

    for idx,file_path in enumerate(all_files):
        label = int(idx / per_dir_num)

        save_dir = src_dir + os.sep + str(label)
        if not os.path.exists(save_dir):
            os.mkdir(save_dir)

        dest_path = save_dir + os.sep + os.path.basename(file_path)
        shutil.move(file_path, dest_path)

# 将一个文件夹下的所有文件随机取出放至一个文件夹
def random_get_img_to_one(data_dir, save_dir, ratio=1/18):
    all_files = get_all_filepath(data_dir, [".jpg"])
    os.makedirs(save_dir, exist_ok=True)

    np.random.shuffle(all_files)
    np.random.shuffle(all_files)
    ratio = int(1 / ratio)

    for idx,file_path in enumerate(all_files):
        if idx % ratio != 0: continue
        shutil.copy(file_path, osp.join(save_dir, osp.basename(file_path)))

# 将一个文件夹下的所有文件按照顺序取出一定比例放至一个文件夹
def order_get_img_to_one(data_dir, save_dir, ratio=1/18):
    all_files = get_all_filepath(data_dir, [".jpg",".png"])
    os.makedirs(save_dir, exist_ok=True)
    all_files =sorted(all_files)
    ratio = int(1 / ratio)

    for idx,file_path in enumerate(all_files):
        if idx % ratio != 0: continue
        shutil.copy(file_path, osp.join(save_dir, osp.basename(file_path)))


def order_get_img_nearly(data_dir, save_dir, filename):
    all_files = get_all_filepath(data_dir, [".jpg"])
    all_files =sorted(all_files)

    find_idx = 0
    for idx,file_path in enumerate(all_files):
        if filename in file_path:
            find_idx = idx
    
    if find_idx > 0:
        for idx in range(find_idx-10, find_idx+10, 1):
            if idx > 0:
                shutil.copy(all_files[idx], osp.join(save_dir, osp.basename(all_files[idx])))



# 临时使用
def print_file_name(dir_name):
    for p in os.listdir(dir_name):
        print(p)

def copy_name(ori_file, name_file):
    with open(name_file, 'r', encoding='utf-8') as rf:
        name_list = rf.readlines()
        new_name_list = [n.split(' ')[0] for n in name_list]

    with open(ori_file, 'r', encoding='utf-8') as rf:
        ori_list = rf.readlines()

    # for l in new_name_list:
    #     print(l.rstrip("\n"))
    # for l in ori_list:
    #     print(l.rstrip("\n"))
    save_list = []
    idx = 0
    flag = False
    for l in ori_list:
        if l.rstrip("\n") == "":
            save_list.append("")
            if flag: 
                flag = False
                idx+=1
        else:
            save_list.append(new_name_list[idx])
            flag = True
    save_ori_file = "_out.".join(ori_file.split("."))
    with open(save_ori_file, "w") as ef:
        for i in save_list:
            ef.write(i + "\n")



def getggg(name_file):
    name = [2019/6/15,2019/6/16,2019/6/17,2019/6/18,2019/6/19,2019/6/20,2019/6/21,2019/6/22,2019/6/23,2019/6/24,2019/6/25,2019/6/26,2019/6/27,2019/6/28,2019/6/29,2019/6/30]
    with open(name_file, 'r', encoding='utf-8') as rf:
        name_list = rf.readlines()
        new_name_list = [n.split(' ')[0] for n in name_list]


# 递归移动指定文件
def move_all_to_one000(all_dir_path, dest_dir, extension=None):
    def check_endswith(new_path, extension_tag):
        for s in extension_tag:
            if new_path.endswith(s):
                return True
        return False

    for f in os.listdir(all_dir_path):
        new_path = all_dir_path + os.sep + f
        if os.path.isfile(new_path) and (extension == None or check_endswith(f, extension)):
            dest_path = dest_dir + os.sep + f
            shutil.move(new_path, dest_path)
        elif os.path.isdir(new_path):
            move_all_to_one000(new_path, dest_dir, extension)

def move_all_to_one_dir(data_dir, dest_dir, extension=[]):
    files = get_all_filepath(data_dir, extension)

    for f in files:
        dst_path = osp.join(dest_dir, osp.basename(f))
        shutil.move(f, dst_path)

def move_all_to_one_dir2(data_dir, via_name="via_region_data.rec.json"):
    sub_dirs = get_last_dir(data_dir, 2)

    for sub_dir in sub_dirs:
        # print("sub_dir: ", sub_dir)

        via_files = get_all_filepath(sub_dir, [via_name])
        if len(via_files) != 0:
            print("sub_dir: ", sub_dir)
            print("via_files: ", len(via_files))

        continue_path = [osp.dirname(v) for v in via_files]
        img_files = get_all_filepath(sub_dir, [".jpg"])
        for i in range(len(img_files)-1, -1, -1):
            for cp in continue_path:
                if cp in img_files[i]:
                    # print("del: ", img_files[i])
                    del img_files[i]
                    break
        # if len(img_files) == 0: print(sub_dir)

        for f in img_files:
            dst_path = osp.join(sub_dir, osp.basename(f))
            if osp.dirname(f) == sub_dir: continue
            shutil.move(f, dst_path)


def remove_same(ori_dir, dest_dir):
    all_files = [ p for p in os.listdir(dest_dir)]

    new_filsdes = [ p for p in os.listdir(ori_dir)]
    for f in new_filsdes:
        for ff in all_files:

            if f == ff:
                os.remove(ori_dir+ os.sep + f)

def crnn_merge_label(data_dir):

    labels_file = get_all_filepath(data_dir, [".txt"])

    total_lines = []
    for f in labels_file:
        dir_name = os.path.basename(os.path.dirname(f))
        with open(f, "r", encoding="utf-8") as rf:
            for l in rf.readlines():
                total_lines.append(dir_name + "/" + l.replace("\n", "") + "\n")
    
    with open(data_dir + os.sep + "labels.txt", "w", encoding="utf8") as wf:
        for l in total_lines:
            wf.write(l)

def remove_corrupt_jpeg(data_dir):
    "Corrupt JPEG data: 2 extraneous bytes before marker 0xd9"
    
    total_files = get_all_filepath(data_dir, [".jpg", ".png"])
    for f in total_files:
        print(f)
        img = cv2.imread(f)     

def remove_gun(data_dir):
    txt_files = get_all_filepath(data_dir, [".txt"])

    for t in txt_files:
        with open(t, "r", encoding="utf8") as rf:
            labels = [int(line[0]) for line in rf.readlines()]
        if 1 in labels: continue

        if osp.exists(t): os.remove(t)
        if osp.exists(t.replace(".txt", ".jpg")): os.remove(t.replace(".txt", ".jpg"))

def remove_file_by_name(data_dir, includes=["***.***"], excludes=["---*---"]):
    def check_name(path):
        for ex in excludes:
            if path.endswith(ex): return False
        for ix in includes:
            if path.endswith(ix): return True
        return False

    for f in get_all_filepath(data_dir):
        if check_name(f): 
            print(f)
            os.remove(f)

def extract_targz(tar_path, dst_path, mode="r:gz"):
    """
    tar : "r"
    tar.gz : "r:gz"
    """
    tar = tarfile.open(tar_path, mode)
    file_names = tar.getnames()
    for file_name in file_names:
        buffer = tar.extractfile(file_name)
        with open(osp.join(dst_path, osp.basename(file_name)), "wb") as wf:
            wf.write(buffer.read())
    tar.close()


def merge_img_and_distribute(data_dir, per_img_in_dir=500):
    ori_dirs = get_last_dir(data_dir, -1)

    img_files = get_all_filepath(data_dir, [".jpg"])
    dir_cnt = len(img_files) // per_img_in_dir
    dir_cnt = dir_cnt if len(img_files) % per_img_in_dir == 0 else dir_cnt+1
    
    # sort by name
    sort_data = [[int(osp.basename(p).split("_")[0]), p] for p in img_files]
    sort_data = sorted(sort_data, key=lambda x: x[0])
    for dir_idx in range(dir_cnt):
        s = dir_idx * per_img_in_dir
        e = (dir_idx+1) * per_img_in_dir
        save_dir = osp.join(data_dir, str(dir_idx))
        os.makedirs(save_dir, exist_ok=True)
        for idx in range(s, e):
            if idx >= len(sort_data): continue
            ori_path = sort_data[idx][1]
            shutil.move(ori_path, osp.join(save_dir, osp.basename(ori_path)))

    for d in ori_dirs:
        shutil.rmtree(d)

def merge_img_and_distribute2(data_dir, per_img_in_dir=500):
    """将一个文件夹下面的所有图片合并重新分配"""
    ori_dirs = get_last_dir(data_dir, -1)

    img_files = get_all_filepath(data_dir, [".jpg"])
    dir_cnt = len(img_files) // per_img_in_dir
    dir_cnt = dir_cnt if len(img_files) % per_img_in_dir == 0 else dir_cnt+1
    
    save_dirs = [osp.join(data_dir, str(i)) for i in range(dir_cnt)]
    for save_dir in save_dirs: os.makedirs(save_dir, exist_ok=True)

    for idx, f in enumerate(img_files):
        choose = idx % dir_cnt
        shutil.move(f, osp.join(save_dirs[choose], osp.basename(f)))

    # for d in ori_dirs:
    #     shutil.rmtree(d)

def merge_img_and_distribute3(data_dir, save_dir, per_img_in_dir=500):
    """"""
    img_files = get_all_filepath(data_dir, [".jpg"])
    
    for idx, f in enumerate(img_files):
        # sub_dir = os.sep.join(osp.basename(f).split("_")[:3])
        # save_path = osp.join(save_dir, sub_dir)
        # os.makedirs(save_path, exist_ok=True)
        # shutil.move(f, osp.join(save_path, osp.basename(f)))
        shutil.move(f, osp.dirname(f) + os.sep + osp.basename(f).replace("fog_", "2021_").replace("rain1_", "2021_").replace("snow_", "2021_"))


def merge_img_and_split4(data_dir, split_str, sort_idx):
    """"""
    img_files = get_all_filepath(data_dir, [".jpg"])
    data_dict = defaultdict(list)
    for img_file in img_files:
        file_name = osp.basename(img_file)
        dir_prefix = file_name.split(split_str)[sort_idx]
        data_dict[dir_prefix].append(img_file)
        
    for k,v in data_dict.items():
        save_dir = osp.join(data_dir, str(k))
        if not osp.exists(save_dir):
            os.makedirs(save_dir, exist_ok=True)
        for f in v:
            shutil.move(f, osp.join(save_dir, osp.basename(f)))


def move_img_by_name(data_dir, save_dir):
    ip_dirs = get_last_dir(data_dir, 0)
    for ip_dir in ip_dirs:
        ip_name = osp.basename(ip_dir)
        img_files = get_all_filepath(ip_dir, [".jpg"])
        for img_path in img_files:
            time_point = int(osp.basename(img_path).split("_")[0]) / 1e3
            ymd = time.strftime('%Y-%m-%d-%H-%M-%S', time.localtime(time_point)).split("-")
            sub_dir = osp.join(str(int(ymd[0])), str(int(ymd[1])), str(int(ymd[2])))
            cur_dir = osp.join(save_dir, sub_dir, ip_name)
            if not osp.exists(cur_dir): os.makedirs(cur_dir)

            dst_path = osp.join(cur_dir, osp.basename(img_path))
            # print(dst_path)
            shutil.move(img_path, dst_path)

def move_img_by_name2(data_dir, save_dir):
        img_files = get_all_filepath(data_dir, [".jpg"])
        for img_path in img_files:
            sub_dir = os.sep.join(osp.basename(img_path).split("_")[:3])
            cur_dir = osp.join(save_dir, sub_dir)
            if not osp.exists(cur_dir): os.makedirs(cur_dir)

            dst_path = osp.join(cur_dir, osp.basename(img_path))
            print(dst_path)
            shutil.move(img_path, dst_path)


def move_img_to_sub_dir(data_dir, img_in_per_dir=50):
    last_dirs = get_last_dir(data_dir)
    for last_dir in last_dirs:

        img_files = get_all_filepath(last_dir, [".jpg"])
        for idx,img_path in enumerate(img_files):
            save_dir = osp.join(last_dir, str(idx // img_in_per_dir))
            if not osp.exists(save_dir): os.makedirs(save_dir)

            dst_path = osp.join(save_dir, osp.basename(img_path))
            print(dst_path)
            shutil.move(img_path, dst_path)


def remove_dir_by_via(data_dir, via_name="via_region_data.json"):
    last_dirs = get_last_dir(data_dir)

    for last_dir in last_dirs:
        via_path = osp.join(last_dir, via_name)
        if not osp.exists(via_path):
            shutil.rmtree(last_dir)
            print(last_dir)


def split_img_to_half(data_dir, dst_dir):
    img_files = get_all_filepath(data_dir, [".jpg", ".png"])
    img_files = sorted(img_files)
    for idx,img_path in enumerate(img_files):
        if idx % 2 == 0: continue

        dst_path = osp.join(dst_dir, osp.basename(img_path))
        shutil.move(img_path, dst_path)

if __name__ == "__main__":
    # all_dir_path = r'C:\Users\vanlance\Downloads\2019.2.14修改'
    # dest_dir = r'C:\Users\vanlance\Downloads\all_img'
    # rename_by_pair(dest_dir)
    # dir_path = sys.argv[1]
    # remove_files_from_dir(dir_path)

    # dir_path = r'E:\work\dataset\test'
    # rename_files_from_dir(dir_path)
    # dir_path = r'C:\Users\vanlance\Desktop\2\2016\out'
    # rename_files_from_dir(dir_path)

    # dir_path = r'E:\work\dataset\video\lbld\11'
    # rename_files_from_dir_save_to_map(dir_path)

    # dir_path = r'E:\work\dataset\video\lbld\11'
    # move_img_to_sub_dir(dir_path)

    # dir_path = r'E:\work\data\fonts\font_bkc_2020_02_04'
    # dest_path = r'E:\work\data\fonts\temp'
    # move_all_to_one000(dir_path,dest_path , extension=["ttf", "otf"])
    # remove_same(dir_path, dest_path)
    # # move_all_to_one_keep_name(dir_path, dest_path, '')
    
    # src_dir = "/mnt/disk2/datasets_kanggle/ocr/plate_data_CCPD/CCPD2019/ccpd_weather"
    # per_dir_num = 500
    # extension_tag = [".jpg"]
    # equal_distribution(src_dir, per_dir_num, extension_tag)

    # dir_path = "/mnt/disk1/vanlance/project/tyre_classify/data_1/0"
    # rename_files_from_dir(dir_path)
    # print_file_name(r'E:\data\food_ocr\food_ocr_image_all\pdf\all\1')

    # a_file = r'E:\work\python_script\ori_list.txt' 
    # name_file = r'E:\work\python_script\name_list.txt' 
    # copy_name(a_file, name_file )

    # path = sys.argv[1]
    # rename_files_from_dir(path)

    # dir_path = sys.argv[1]
    # dest_dir = dir_path
    # move_all_to_one2(dir_path, dest_dir)

    # dir_path = sys.argv[1]
    # replace_char(dir_path)

    # data_dir = sys.argv[1]
    # move_all_to_one_dir2(data_dir)

    # data_dir = "/home/swls/work_dir/ocr/code/syn/syn_plate_300k"
    # crnn_merge_label(data_dir)

    # std_dir_path="/home/swls/work_dir/lib/ffmpeg_build"
    # remove_dir="/usr/local"
    # remove_same_files(std_dir_path, remove_dir)

    # file_dir = sys.argv[1]
    # for f in get_all_filepath(file_dir, ["via_region_data.json"]):
    #     print(f)
    #     os.rename(f, f.replace("via_region_data.json", "via_region_data.old.json"))

    # data_dir = sys.argv[1]
    # remove_corrupt_jpeg(data_dir)

    # data_dir = sys.argv[1]
    # remove_gun(data_dir)

    # data_dir = "/home/xc/work/code/paddle/train_data/det/fire/images/2021-09-29/192.168.202.64_01_20210929160003654_1"
    # save_dir = "/home/xc/work/code/paddle/train_data/det/fire/images/2021-09-29/192.168.202.64_01_20210929160003654_1/total"
    # random_get_img_to_one(data_dir, save_dir, ratio=1/18)

    # data_dir = "/home/sunjie/work/dataset/青海保密文档-切勿外传/doc/xnbbdz_img"
    # save_dir = "/home/sunjie/work/dataset/青海保密文档-切勿外传/doc/test_img4"
    # order_get_img_to_one(data_dir, save_dir, ratio=1/50)


    # data_dir = "/home/xc/work/code/paddle/train_data/det/car/images/2021/10/18"
    # data_dir = sys.argv[1]
    # auto_split_data(data_dir)

    # data_dir = sys.argv[1]
    # merge_img_and_distribute(data_dir, 300)

    # data_dir = sys.argv[1]
    # merge_img_and_distribute2(data_dir, 2000)


    # data_dir = sys.argv[1]
    # save_dir = sys.argv[2]
    # merge_img_and_distribute3(data_dir, save_dir)

    # # 按照特征将所有图片分门别类存放 split
    # data_dir = sys.argv[1]
    # split_str = sys.argv[2]
    # sort_idx = int(sys.argv[3])
    # # python filesystem.py ./ . 0
    # merge_img_and_split4(data_dir, split_str, sort_idx)
    
    # data_dir = "/home/xc/work/code/paddle/train_data/det/car/images/2021/8/0"
    # save_dir = "/home/xc/work/code/paddle/train_data/det/car/images"
    # move_img_by_name(data_dir, save_dir)
    # data_dir = "/home/xc/work/code/paddle/train_data/cls/weather/images/leishan/2022/picture"
    # save_dir = "/home/xc/work/code/paddle/train_data/cls/weather/images/leishan"
    # move_img_by_name2(data_dir, save_dir)

    # data_dir = sys.argv[1]
    # # includes=["big_truck.jpg", "car_stop.jpg", "move_slow.jpg", 
    # #           "converse_move.jpg", "alarm_FireWarning.jpg", 
    # #           "SandDetection_log_ori.jpg", "SandDetection_log_mask.jpg",
    # #           ]
    # includes = ["via_region_data.json"]
    # remove_file_by_name(data_dir, includes)

    # data_dir = "/mnt/data/sj/images/2021/11"
    # remove_dir_by_via(data_dir)

    # data_dir = "/home/sunjie/work/dataset/青海保密文档-切勿外传/doc/img.14"
    # dst_dir = "/home/sunjie/work/dataset/青海保密文档-切勿外传/doc/img.15"
    # split_img_to_half(data_dir, dst_dir)

    # path1 = "/home/xc/weather/images/pic2/rain1"
    # path2 = "/home/xc/python/my_/all/weather/pic2"
    # fog3 = set([osp.basename(p) for p in get_all_filepath(path1, [".jpg"])])

    # cnt = 0
    # for p in get_all_filepath(path2, [".jpg"]):
    #     name = osp.basename(p)
    #     if name in fog3:
    #         cnt += 1
    #         # os.remove(p)
    # print("cnt:", cnt)


    # # 将一个文件夹下的所有文件按照顺序取出一定比例放至一个文件夹
    # mode = sys.argv[1]
    # data_dir = "/home/sunjie/0/0926/34020000001320000027"
    # if mode == "1":
    #     save_dir = "/home/sunjie/0/0926/ttt"
    #     order_get_img_to_one(data_dir, save_dir, ratio=1/7)
    # # 将此文件名前后10张图片取出
    # if mode == "2":
    #     save_dir = "/home/sunjie/0/0926"
    #     filename = sys.argv[2]
    #     order_get_img_nearly(data_dir, save_dir, filename)

    
    pass